package main

import (
	"context"
	"encoding/csv"
	"encoding/json"
	"fmt"
	"log"
	"os"
	"regexp"
	"strconv"
	"strings"
	"time"

	mysql "aqielife.cn/spider/db"
	"aqielife.cn/spider/utils"
	colly "github.com/gocolly/colly/v2"
	"github.com/gocolly/colly/v2/debug"
	"go.mongodb.org/mongo-driver/bson/primitive"
	"go.mongodb.org/mongo-driver/mongo"
	"go.mongodb.org/mongo-driver/mongo/options"
)

var (
	dberr      error
	client     *mongo.Client
	db         *mongo.Database
	collection *mongo.Collection
	id         primitive.ObjectID
	Datas       []interface{} 
	IDs         []int
)

// Article 抓取blog数据
type Article struct {
    ID       int    `json:"id,omitempty"`
    Title    string `json:"title,omitempty"`
    URL      string `json:"url,omitempty"`
    Created  string `json:"created,omitempty"`
    Reads    string `json:"reads,omitempty"`
    Comments string `json:"comments,omitempty"`
    Feeds    string `json:"feeds,omitempty"`
}

// 数据持久化
func csvSave(fName string, data []Article) error {
    file, err := os.Create(fName)
    if err != nil {
        log.Fatalf("Cannot create file %q: %s\n", fName, err)
    }
    defer file.Close()
    writer := csv.NewWriter(file)
    defer writer.Flush()

    writer.Write([]string{"ID", "Title", "URL", "Created", "Reads", "Comments", "Feeds"})
    for _, v := range data {
        writer.Write([]string{strconv.Itoa(v.ID), v.Title, v.URL, v.Created, v.Reads, v.Comments, v.Feeds})
    }
    return nil
}


//插入mongo一条数据库
func InsertRecord(client *mongo.Client, collect *mongo.Collection,data Article) (insertID primitive.ObjectID) {
 
	collect = client.Database("aqie").Collection("article")
	insertRest, err := collect.InsertOne(context.TODO(), data)
	if err != nil {
		fmt.Println(err)
		return
	}
	insertID = insertRest.InsertedID.(primitive.ObjectID)
	// fmt.Println("插入Id：",insertID.Hex())
	return insertID
}

// 批量插入 mongodb
func InsertMany(client *mongo.Client, data []interface{}) {
	collection := client.Database("aqie").Collection("article")
	ids, _ := collection.InsertMany(context.TODO(),data)
	if ids != nil {
		// fmt.Printf("mongo:[%s]", ids)
	}
}

func main() {
	start := time.Now() // 获取当前时间
	ctx, cancelFunc := context.WithCancel(context.TODO())
	time.AfterFunc(10*time.Second, func() {
		cancelFunc()
	})
	//数据库连接
	if client, dberr = mongo.Connect(ctx, options.Client().ApplyURI("mongodb://localhost:27017").SetConnectTimeout(5*time.Second)); dberr != nil {
		fmt.Print(dberr)
		return
	}

	articles := make([]Article, 0, 200)
	mUrl := "https://learnku.com/blog/pardon"
	c := colly.NewCollector(
		colly.Debugger(&debug.LogDebugger{}),
		colly.AllowedDomains("learnku.com"),
		// 防止页面重复下载
        colly.CacheDir("./learnku_cache"),
	)
	//发送请求之前的执行函数
	c.OnRequest(func(r *colly.Request) {
		log.Println("OnRequest")
	})
	//发送请求错误被回调
	c.OnError(func(_ *colly.Response, err error) {
		fmt.Print(err)
	})

	//响应请求之后被回调
	c.OnResponse(func(r *colly.Response) {
		fmt.Println("Response body length：", len(r.Body))
	})
	

	c.OnHTML("div.blog-article-list > .event", func(e *colly.HTMLElement) {
		article := Article{
            Title: e.ChildText("div.content > div.summary"),
            URL:   e.ChildAttr("div.content a.title", "href"),
            Feeds: e.ChildText("div.item-meta > a:first-child"),
        }
		// 查找同一集合不同子项
        e.ForEach("div.content > div.meta > div.date>a", func(i int, el *colly.HTMLElement) {
            switch i {
            case 1:
                article.Created = el.Attr("data-tooltip")
            case 2:
                // 用空白切割字符串
                article.Reads = strings.Fields(el.Text)[1]
            case 3:
                article.Comments = strings.Fields(el.Text)[1]
            }
        })

		// 正则匹配替换,字符串转整型
        article.ID, _ = strconv.Atoi(regexp.MustCompile(`\d+`).FindAllString(article.URL, -1)[0])
		
		// 根据文章id 去重
		if(!utils.IsContain(IDs, article.ID)){
			IDs = append(IDs, article.ID)
			articles = append(articles, article)
			Datas = append(Datas, article)
		}
	})

	// 下一页
    c.OnHTML("a[href].page-link", func(e *colly.HTMLElement) {
        e.Request.Visit(e.Attr("href"))
    })

	//在OnHTML之后被调用
	c.OnScraped(func(r *colly.Response) {
		fmt.Println("Finished", r.Request.URL)
	})
	//这里是执行访问url
	c.Visit(mUrl)

	// 1. 输出到csv
	// csvSave("pardon.csv", articles)

	// 2. 保存到 mongodb
	// InsertMany(client, Datas)

	// 3. 保存到mysql
	if err := mysql.DB.Create(&articles).Error; err != nil {
		log.Printf("db.Create err : %v", err)
	} else {
		log.Println("批量新增成功: ", len(articles))
	}

	//延时关闭数据库连接
	sqlDB, _ := mysql.DB.DB()
	defer sqlDB.Close()

	enc := json.NewEncoder(os.Stdout)
	enc.SetIndent("", "  ")
	// enc.Encode(articles)
	// 显示收集器的打印信息
    // log.Println(c)

	elapsed := time.Since(start)
	fmt.Println("该函数执行完成耗时：", elapsed)
}
